# coding:utf-8

from pyspark.sql import SparkSession
from pyspark.sql import Row
import matplotlib.pyplot as plt
import matplotlib as mpl
import sys
import numpy as np

spark = SparkSession.builder \
    .appName("Python Spark SQL basic example") \
    .getOrCreate()

sc = spark.sparkContext

userrdd = sc.textFile("C:\\Users\\ljb\\Desktop\\user_test.txt").map(lambda line:line.split(","))

df = userrdd.map(lambda fields: Row(userid = fields[0],age = int(fields[1]),gender = fields[2],
                                    occupation = fields[3],zipcode = fields[4]))
schemauser = spark.createDataFrame(df)
schemauser.createOrReplaceTempView("user")
# age = spark.sql("SELECT * FROM user")

# ages = age.rdd.map(lambda p:p.age).collect()


# plt.figure("hist")
# plt.hist(ages,bins=25,color="lightblue",normed=True)
# plt.show()

count_occp = spark.sql("SELECT occupation,count(occupation) as cnt FROM user GROUP BY occupation ORDER BY cnt")
# count_occp.show(21)

x_axis = count_occp.rdd.map(lambda p:p.occupation).collect()
y_axis = count_occp.rdd.map(lambda p:p.cnt).collect()

# 指定默认字体 SimHei为黑体,解决画图中文字符为乱码的问题
mpl.rcParams['font.sans-serif']=['SimHei']
# 使负号 "-" 能够正常显示
mpl.rcParams['axes.unicode_minus']=False

pos = np.arange(len(x_axis))
width = 1.0
ax = plt.axes()
# 设置 x 轴刻度
ax.set_xticks(pos + (width / 2))
# 在对应的刻度上打上标签
ax.set_xticklabels(x_axis)
plt.bar(pos,y_axis,width,color="orange")
# x 轴刻度标签旋转 30度
plt.xticks(rotation=30)
# 设置 x 轴坐标标签
plt.xlabel("x + width)")
# 设置 y 抽坐标标签
plt.ylabel("职业数量")

fig = plt.gcf()
fig.set_size_inches(16,10)
plt.show()
